# To change this template, choose Tools | Templates
# and open the template in the editor.

import formatter
import htmllib
import urllib

from sitemapgen.linksextractor import LinksExtractor

class Generator:

	unparsed_links = []
	current_links = []
	parsed_links = []
	def __init__(self, url):
		if not url.startswith("http://"):
			url = "http://" + url
		if url.endswith("/"):
			url = url[0:len(url) - 1]
			
		self.base_url = url
		self.unparsed_links.append(url)
		self.get_links()
		
	def get_links(self):
		format = formatter.NullFormatter()
		htmlparser = LinksExtractor(format)
		if not len(self.unparsed_links) == 0:
			self.current_links = []
			for link in self.unparsed_links:
				print "Parsing: " + link
				data = urllib.urlopen(link)
				htmlparser.feed(data.read())      # parse the file saving the info about links
				htmlparser.close()
				self.current_links = htmlparser.get_links() # get the hyperlinks list
				self.parsed_links.append(link)
				self.unparsed_links.remove(link)
				self.sort_links()
			self.get_links()

	def sort_links(self):
		for link in self.current_links:
			link = self.format_link(link)
			if not link in self.parsed_links:
				if not link in self.unparsed_links:

					if not link == None:
						self.unparsed_links.append(link)

	def format_link(self, link):
		htm_formats = [".htm",".php",".txt",".asp",".cgi",".pl"]
		image_types = [".gif", ".jpg",".png"]
		is_hf = False

		for image_type in image_types:
			if image_type in link:
				return None
		if "." in link:
			for hf in htm_formats:
				if hf in link:
					is_hf = True
			if not hf:
				return None
		if link.endswith('/'):
			link = link[0: len(link) - 1]
		if link.startswith('#'):
			return None
		if link.startswith('/'):
			return self.base_url + link
		if self.base_url in link:
			return link
		if link.startswith("http://") and not self.base_url in link:
			return None

	def create_xml(self, output):
		fh = open(output,"w")
		fh.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n")
		fh.write("<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://www.sitemaps.org/schemas/sitemap/0.9\thttp://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd\">")
		for link in self.parsed_links:
			fh.write("<url><loc>" + link + "</loc></url>")
		fh.write("</urlset>")
		fh.close()

	def gzip_file(self, output):
		import gzip
		xh = open(output, "rb")
		gh = gzip.open(output + ".gz", "wb")
		gh.writelines(xh)
		xh.close()
		gh.close()

	def create_txt(self, output):
		fh = open(output,"w")
		for link in self.parsed_links:
			fh.write(link + "\n")
		fh.close()

	def create_html(self, output):
		fh = open(output,"w")
		fh.write("<HTML><HEAD><TITLE>SITEMAP " + self.base_url + "</TITLE></HEAD>" )
		fh.write("<BODY><UL>")
		for link in self.parsed_links:
			fh.write("<LI><A HREF=\"" + link + "\">" + link + "</A>")
		fh.write("</UL></BODY></HTML>")
		fh.close()
		
